/*************************************************************************
    > File Name    : PageLibPreprocessor.cc
    > Author       : LiuLei
    > Mail         : 2261459486@qq.com 
    > Created Time : 2022年11月29日 星期二 16时14分46秒
 ************************************************************************/

#include "../../include/WebProcessor/PageLibPreprocessor.h"
#include <math.h>
#include <fstream>
#include <iostream>
#include <sstream>
using std::ifstream;
using std::cout;
using std::endl;
using std::istringstream;
using std::make_pair;
using std::ofstream;

PageLibPreprocessor::PageLibPreprocessor(Configuration &conf)
: _config(conf.getConfigMap())
{
    /* // 注意：自定义类型的元素存放在vector中，扩容最好不要使用resize */
    /* // 除非有自定义的拷贝构造函数 */
    /* /1* _pageLib.resize(1024); *1/ */
    /* _pageLib.reserve(1024); */
}

PageLibPreprocessor::~PageLibPreprocessor()
{

}

void PageLibPreprocessor::doProcess()
{
    this->readInfoFromFile();
    this->cutRedundantPages();
    this->buildInvertIndexTable();
    this->storeOnDisk();
}

void PageLibPreprocessor::readInfoFromFile()
{
    // 加载网页库
    cout << "Loading PageLib, please wait..." << endl;
    string pageLibPath = _config.at("webpage");
    ifstream ifs1;
    ifs1.open(pageLibPath);
    if(!ifs1.good())
    {
        cout << "Open " << pageLibPath << " error in PageLibPreprocessor::readInfoFromFile()!" << endl;
        return ;
    }

    // 写入网页库
    string doc;
    while(getline(ifs1, doc))
    {
        WebPage page(doc, Configuration::getInstance(), _jieba);
        /* // 考虑扩容 */
        /* if(_pageLib.size() == _pageLib.capacity()) */
        /* { */
        /*     /1* _pageLib.resize(_pageLib.capacity() * 2); *1/ */
        /*     _pageLib.reserve(_pageLib.capacity() * 2); */
        /* } */
        _pageLib.push_back(page);
    }
    ifs1.close();

    // 加载偏移库
    cout << "Loading OffsetLib, please wait..." << endl;
    string offsetLibPath = _config.at("newoffset");
    ifstream ifs2;
    ifs2.open(offsetLibPath);
    if(!ifs2.good())
    {
        cout << "Open " << pageLibPath << " error in PageLibPreprocessor::readInfoFromFile()!" << endl;
        return ;
    }

    // 写入偏移库
    string offsetData;
    while(getline(ifs2, offsetData))
    {
        int docId;
        int startPosition;
        int docSize;
        istringstream iss(offsetData);
        if(iss >> docId)
        {

        }
        else
        {
            cout << ">> DocId load failed in PageLibPreprocessor::readInfoFromFile()!" << endl;
            return ;
        }

        if(iss >> startPosition)
        {

        }
        else
        {
            cout << ">> StartPosition load failed in PageLibPreprocessor::readInfoFromFile()!" << endl;
            return ;
        }

        if(iss >> docSize)
        {

        }
        else
        {
            cout << ">> DocSize load failed in PageLibPreprocessor::readInfoFromFile()!" << endl;
            return ;
        }
        _offsetLib[docId] = make_pair(startPosition, docSize);
    }
    ifs2.close();
}

void PageLibPreprocessor::cutRedundantPages()
{
    cout << "Before cut, the number is " << _pageLib.size() << endl;
    cout << "Cutting redundant pages, please wait..." << endl;

    // 外层for循环作为对比基准
    for(list<WebPage>::iterator it1 = _pageLib.begin(); it1 != _pageLib.end(); ++it1)
    {
        list<WebPage>::iterator pit = it1;

        if(++pit != _pageLib.end())
        {
            // 内层for循环定位到基准迭代器的下一个位置
            /* for(list<WebPage>::iterator it2 = pit; it2 != _pageLib.end(); ++it2) */
            // 不能在这里直接进行++it2，先要判断情况

            for(list<WebPage>::iterator it2 = pit; it2 != _pageLib.end(); )
            {
                if((*it1) == (*it2))
                {
                    _offsetLib.erase((*it2).getDocId());
                    _pageLib.erase(it2++);
                    // 因为erase删除的是迭代器的前一个位置
                }
                else
                {
                    ++it2;
                }
            }
        }
    }

    cout << "After cut, the number is " << _pageLib.size() << endl;
}

void PageLibPreprocessor::buildInvertIndexTable()
{
    cout << "Building invertindex table, please wait..." << endl;
    long wholeDocNum = _pageLib.size();
    
    // 建立总词频统计
    map<string, int> documentFrequency;
    for(auto it = _pageLib.begin(); it != _pageLib.end(); ++it)
    {
        // 获取文档的词频统计
        map<string, int> curDocFrequency = (*it).getWordsMap();

        // 遍历该文档的词频统计
        for(auto cit = curDocFrequency.begin(); cit != curDocFrequency.end(); ++cit)
        {
            string key = (*cit).first;
            documentFrequency[key] += (*cit).second;
        }
    }

    // 建立每个文章每个单词的权重库
    map<int, vector<pair<string, double>>> weightMap;
    for(auto it = _pageLib.begin(); it != _pageLib.end(); ++it)
    {
        int docId = (*it).getDocId();

        // 获取谋篇文章某个单词的词频
        map<string, int> curDocFrequency = (*it).getWordsMap();
        for(auto cit = curDocFrequency.begin(); cit != curDocFrequency.end(); ++cit)
        {
            // 某个词在文章中出现的次数
            string word = (*cit).first;
            int termFrequency = (*cit).second;

            // 逆文档率
            double inverseDcoFrequency = log2(wholeDocNum / documentFrequency.at(word) + 1);
            // 权重
            double weight = termFrequency * inverseDcoFrequency;
            weightMap[docId].push_back(make_pair(word, weight));
        }
    }

    // 归一化处理
    for(auto it = weightMap.begin(); it != weightMap.end(); ++it)
    {
        // 计算此文档的归一量
        double wholeFrequency = 0.0;
        for(auto &frequency : (*it).second)
        {
            wholeFrequency += frequency.second * frequency.second;
        }
        wholeFrequency = ::sqrt(wholeFrequency);

        // 计算此文档的倒排索引权重
        for(auto &frequency : (*it).second)
        {
            // 计算最终权重值并保留到成员变量中
            double realWeight = frequency.second / wholeFrequency;
            _invertIndexTable[frequency.first].push_back(make_pair(it->first, realWeight));
        }
    }
}

void PageLibPreprocessor::storeOnDisk()
{
    cout << "Storing libs, please wait..." << endl;

    // 存入去重的网页偏移库
    ofstream ofs1;
    string offsetPath = _config.at("newoffset");
    ofs1.open(offsetPath, std::ios::out);
    if(!ofs1.good())
    {
        cout << "Open " << offsetPath << " error in PageLibPreprocessor::storeOnDisk()!" << endl;
        return ;
    }
    for(auto it = _offsetLib.begin(); it != _offsetLib.end(); ++it)
    {
        ofs1 << (*it).first << "  " << (*it).second.first << " " << (*it).second.second << endl;
    }
    cout << "Finish storing offsetLib!" << endl;
    ofs1.close();

    // 存入倒排索引库
    ofstream ofs2;
    string inverseIndexPath = _config.at("invertindex");
    ofs2.open(inverseIndexPath, std::ios::out);
    if(!ofs2.good())
    {
        cout << "Open " << inverseIndexPath << " error in PageLibPreprocessor::storeOnDisk()!" << endl;
        return ;
    }
    for(auto &mapElem : _invertIndexTable)
    {
        ofs2 << mapElem.first << "   ";
        for(auto &vectorElem : mapElem.second)
        {
            ofs2 << vectorElem.first << " " << vectorElem.second << " ";
        }
        ofs2 << endl;
    }
    cout << "Finish storing inverseIndexLib!" << endl;
    ofs2.close();
}
